#!/usr/bin/env python

########################################################
# Part of MILIM NIRDAFOT project.
# Written by H. Magal, 2014
# See details on the project at the site:
#   https://sites.google.com/site/nirdafotdownloads/home
########################################################

# Find all files with the pattern PLxxxxxx.csv, read the one line content, and sort all lines
# into a single sorted list. The sort is done while removing NIKUD characters, and leaving the rest
# for the sort process. The list itself holds the full lines with NIKUD characters as well.
# The final sorted list is written into output.csv file.

import codecs
import string
import os
import re
import sys
from hmHebrew import *

# next function returns a list of all files matching the pattern we need (PLxxxx.csv)
def search_CSV_files ():
    fileNames = list ()                             # create an empty list
    for f in os.listdir ('.'):
        if not (re.match ('PL[0-9]+\.csv', f) is None): # a match was found
            fileNames.append (f)                    # add the name to the list
    return fileNames

# next function reads a line from every file and it creates a list of all words for each line.
# each such a list has as the LAST word the filename from which that line was read !
# The returned value is the list of all lists.
def aggregate_all_lines (filenames):
    biglist = list ()
    groups = 0
    words = 0
    flist = open ('files.txt', 'w')                 # write how many lines are at each file
    maXconcurrentNumber = 0
    for f in filenames:
        fin = codecs.open (filename=f, encoding='utf_16')
        lin = fin.readlines ()
        fin.close()
        flistst = f + ': ' + str (len(lin)) + ' lines\n'
        flist.write (flistst)
        for l in lin:                               # loop through all lines
            l1 = l.rstrip ('\n\r')                  # remove all trailing CR, LF chars
            l1 = l1.rstrip (',')                    # remove all trailing ',' (to get rid of empty words)
            lis = l1.split (',')                    # split line to a list with comma as separator
            concurrentNumber = lis.pop ()           # fetch and remove the last item which is the no-concurrent-number
            lis.sort (compare_words_without_NIKUD)  # sort list according to first word without NIKUD (in place)
            words += len (lis)
            lis.append (concurrentNumber)           # restore the no-concurrent-number after sort
            lis.append (f)                          # file name is the last element of every list
            biglist.append (lis)
            groups += 1                             # count groups count
            try:
                concurrentNumber = int (concurrentNumber) # from string to int
            except:
                print '   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'
                print '   We encountered error at the non-concurrent-number. File=' + f
                print '   <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'
                sys.exit(1)

            if concurrentNumber > maXconcurrentNumber:
                maXconcurrentNumber = concurrentNumber
    flistst = 'Highest Non.Concurrent.Number:' + str (maXconcurrentNumber) +'\n'
    flist.write (flistst)
    flist.close ()
    return (biglist, groups, words)

# next function writes output files (with/without NIKUD)
def write_target_CSV_file (lines, oname):
    fout = codecs.open (filename=oname, mode='w', encoding='utf_16')
    for l in lines:
        first_word = True
        for w in l:                                 # l is a list itself, we loop on its words
            if first_word is False:
                fout.write (',')                    # write comma before second word and on
            w2 = w.strip ('"')                      # remove quotation marks from both ends of word
            fout.write (w2)
            first_word = False
        fout.write ('\n')
    fout.close ()


# next function writes output file without NIKUD
# and only for NO-CONCURRENT entries
def write_target_CSV_file_No_Concurrent_Only (lines, oname):
    fout = codecs.open (filename=oname, mode='w', encoding='utf_16')
    fout.write ('0,')                               # first line is an explanation to user
    fout.write ('Sort this file,According to,Column A,B4 using it\n')
    for l in lines:
        if l[-2] == '0':
            continue                                # skip printing of line if the non-concurrent number is 0
        fout.write (l[-2].strip ('"'))              # write concurrent number as first arg (in addition to its normal place)
        for w in l:                                 # l is a list itself, we loop on its words
            w2 = w.strip ('"')                      # remove quotation marks from both ends of word
            fout.write (','+w2)
        fout.write ('\n')
    fout.close ()


# create a translation table which includes only the NIKUD characters, to enable deletion of them !
# we include in the deletion_table also the '"' char, to enable proper comparison of words,
# those with and without '"' around them.
NIKUD_characters = SHEVA + HATAF_SEGOL + HATAF_PATAH + HATAF_QAMATS + HIRIQ + TSERE +\
                   SEGOL + PATAH + QAMATS + HOLAM + HOLAM_HASER_FOR_VAV + QUBUTS +\
                   DAGESH_OR_MAPIQ + SHIN_DOT + SIN_DOT + PUN_GERESH + PUN_GERSHAYIM + '"'
deletion_table = dict ((ord(char), None) for char in NIKUD_characters)

# reurn a string without the NIKUD characters
def NoNikud (st):
    s1 = st.translate (deletion_table)       # remove NIKUD characters
    return s1


# compare list according to words without NIKUD
def compare_words_without_NIKUD (w1, w2):
    w1NoNik = NoNikud (w1)
    w2NoNik = NoNikud (w2)
    if w1NoNik == w2NoNik:
        return 0
    if w1NoNik > w2NoNik:
        return 1
    return -1


# sort list according to first word without NIKUD
def compare_without_NIKUD (l1, l2):
    w1 = l1[0]                                    # take first word of every list
    w2 = l2[0]
    return compare_words_without_NIKUD (w1, w2)


# Create a second list where all NIKUD characters are removed. We create this list to enable searching
# in the csv file (this list is going to written to file too)
def remove_NIKUD_from_list (inlist):
    ol = list()                           # empty output list
    for l in inlist:                      # loop on lines
        nl = list()                       # form empty output line
        wcnt = 0
        for w in l:                       # loop on words of each line
            if wcnt < (len (l) -1):       # The last word is file name, don't send it to NoNikud()
                w1 = NoNikud (w)
            else:
                w1 = w
            wcnt += 1
            nl.append (w1)
        ol.append (nl)
    return ol

###### Main code ######
def main():
    filenames = search_CSV_files ()                     # a list of all files with the requested pattern (Plxxxxxx.csv)
    allLines, groups, words = aggregate_all_lines (filenames) # This is the Unsorted list of all lines
    allLines.sort (compare_without_NIKUD)               # sort list according to first word without NIKUD
    write_target_CSV_file (allLines, 'output.csv')      # write output file
    print 'output.csv was created. ' + repr(groups) + ' groups. ' + repr(words) + ' words.'
    allLinesNN = remove_NIKUD_from_list (allLines)
    write_target_CSV_file (allLinesNN, 'outputNN.csv')  # write auxiliary output file
    write_target_CSV_file_No_Concurrent_Only (allLinesNN, 'outputEX.csv')

if __name__ == '__main__':
    main ()